#!/bin/sh
DOWNLOAD_PATH=/opt/download/
INSTALL_PATH=/usr/local
WORKSPACE=$(dirname $(realpath $0))
STRUCTURE=$(uname -m)

download_pkg(){
	[ -d "$DOWNLOAD_PATH" ] || mkdir $DOWNLOAD_PATH
	cd $DOWNLOAD_PATH
	wget https://repo.huaweicloud.com/apache/hadoop/core/hadoop-2.7.6/hadoop-2.7.6.tar.gz
	wget https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper-3.5.9/apache-zookeeper-3.5.9-bin.tar.gz
	wget https://repo.huaweicloud.com/apache/kafka/2.2.2/kafka_2.11-2.2.2.tgz
	wget https://repo.huaweicloud.com/apache/hive/hive-2.3.5/apache-hive-2.3.5-bin.tar.gz
	wget https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-java-5.1.40.tar.gz
	
	if [[ $STRUCTURE =~ "x86_64" ]];then
		wget https://downloads.mysql.com/archives/get/p/23/file/mysql-5.7.27-linux-glibc2.12-x86_64.tar.gz
	elif [[ $STRUCTURE =~ "aarch64" ]];then
		wget https://obs.cn-north-4.myhuaweicloud.com/obs-mirror-ftp4/database/mysql-5.7.27-aarch64.tar.gz
	else
		echo "System architecture not found, please try to install manually!!!"
	fi

    cd $WORKSPACE
	git clone https://gitee.com/shikaizhou/streamsets.git
}
install_depends()
{
	yum -y -q install tar git rpm-build spectool 'libaio*' libatomic
}

env_enable()
{
	source /etc/profile > /dev/null
}

install_jdk()
{
	yum install -y -q java-1.8.0-openjdk java-1.8.0-openjdk-devel
	JAVA_HOME=$(dirname $(dirname $(realpath $(which java))))
	echo "export JAVA_HOME=$JAVA_HOME" >> /etc/profile
	echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile
	env_enable
}

install_hadoop()
{
	cd $INSTALL_PATH
	tar -zxf $DOWNLOAD_PATH/hadoop-2.7.6.tar.gz -C $INSTALL_PATH/
	mv hadoop-2.7.6 hadoop
	cd $INSTALL_PATH/hadoop/etc/hadoop
    \cp -f $WORKSPACE/streamsets/scripts/conf/core-site.xml \
           $WORKSPACE/streamsets/scripts/conf/hdfs-site.xml \
           $WORKSPACE/streamsets/scripts/conf/mapred-site.xml $INSTALL_PATH/hadoop/etc/hadoop/ \
           $WORKSPACE/streamsets/scripts/conf/hadoop-env.sh $INSTALL_PATH/hadoop/etc/hadoop/

    cat > /etc/profile <<-EOF
    export HADOOP_HOME=/usr/local/hadoop
    export HADOOP_INSTALL=$HADOOP_HOME
    export HADOOP_COMMON_HOME=$HADOOP_HOME
    export HADOOP_MAPRED_HOME=$HADOOP_HOME
    export HADOOP_HDFS_HOME=$HADOOP_HOME
    export YARN_HOME=$HADOOP_HOME
    export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
    export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin
    export HDFS_NAMENODE_USER=root
    export HDFS_DATANODE_USER=root
    export HDFS_SECONDARYNAMENODE_USER=root
    export YARN_RESOURCEMANAGER_USER=root
    export YARN_NODEMANAGER_USER=root
    EOF
    \cp -f $WORKSPACE/streamsets/scripts/conf/hostname /etc/
    hostname $(cat /etc/hostname)
	env_enable
}

set_ssh()
{
	cd ~/.ssh
	ssh-keygen -t rsa -N '' -f id_rsa -q
    sleep 3 # why? add comment or remove
	cat id_rsa.pub >> authorized_keys
    chmod 600 authorized_keys
}

start_hadoop()
{
	set_ssh
	env_enable
	hdfs namenode -format
	start-all.sh
	jps
}

install_mysql()
{
	cd $INSTALL_PATH
    # yum install mysql default install mysql8.x
    # we need mysql5.7
	if [[ $STRUCTURE =~ "x86_64" ]];then
		tar -xf $DOWNLOAD_PATH/mysql-5.7.27-linux-glibc2.12-x86_64.tar.gz -C $INSTALL_PATH/
		mv mysql-5.7.27-linux-glibc2.12-x86_64 mysql
	elif [[ $STRUCTURE =~ "aarch64" ]];then
		tar -xf $DOWNLOAD_PATH/mysql-5.7.27-aarch64.tar.gz -C $INSTALL_PATH/
		mv mysql-5.7.27-aarch64 mysql
        \cp -rf $INSTALL_PATH/mysql/extra/lib* /usr/lib64/
	else
		echo "System architecture not found, please try to install manually!!!"
	fi
	
	groupadd -r mysql && useradd -r -g mysql -s /sbin/nologin -M mysql
	mkdir -p $INSTALL_PATH/mysql/logs
	chown -R mysql:mysql $INSTALL_PATH/mysql
	ln -sf $INSTALL_PATH/mysql/my.cnf /etc/my.cnf
	mv /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so.6.old
	ln -s /usr/lib64/libstdc++.so.6.0.24 /usr/lib64/libstdc++.so.6
	\cp -rf $INSTALL_PATH/mysql/support-files/mysql.server /etc/init.d/mysqld
	chmod +x /etc/init.d/mysqld
	/usr/lib/systemd/systemd-sysv-install enable mysqld
	echo 'export MYSQL_HOME=/usr/local/mysql' >> /etc/profile
	echo 'export PATH=$PATH:$MYSQL_HOME/bin'>> /etc/profile
	env_enable
}

start_mysql()
{
	mysqld --initialize-insecure --user=mysql --basedir=$INSTALL_PATH/mysql --datadir=$INSTALL_PATH/mysql/data
	systemctl start mysqld
	mysql -u root -e "
	use mysql;
	update mysql.user set authentication_string=password('root') where User='root';
	create database streamsets;
	GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'root' WITH GRANT OPTION;
	grant all privileges on streamsets.* to root;
	FLUSH PRIVILEGES;
	use streamsets;
	create table test(id int primary key auto_increment,name varchar(50) not null,age int not null);
	create table test_target(id int primary key auto_increment,name varchar(50) not null,age int not null);
	insert into test(id,name,age) values(1,'zhou',18);
	insert into test(id,name,age) values(2,'abc',20);
	insert into test(id,name,age) values(3,'sss',11);
	insert into test(id,name,age) values(4,'aaa',12);
	"
}

install_zookeeper()
{
	cd $INSTALL_PATH
	tar -zxf $DOWNLOAD_PATH/apache-zookeeper-3.5.9-bin.tar.gz -C $INSTALL_PATH/
	mv apache-zookeeper-3.5.9-bin zookeeper
	cd $INSTALL_PATH/zookeeper
	mkdir data
	mkdir logs
	cd conf
	touch zoo.cfg
	echo 'tickTime=2000
	initLimit=10
	syncLimit=5
	dataDir=/usr/local/zookeeper/data
	clientPort=2181
	'>>zoo.cfg
}

start_zookeeper()
{
	cd $INSTALL_PATH/zookeeper/bin
	./zkServer.sh start
}

install_kafka()
{
	cd $INSTALL_PATH
	tar -zxf $DOWNLOAD_PATH/kafka_2.11-2.2.2.tgz -C $INSTALL_PATH/
	mv kafka_2.11-2.2.2 kafka
}

start_kafka()
{
	cd $INSTALL_PATH/kafka/bin
	nohup ./kafka-server-start.sh ../config/server.properties &
}

install_hive()
{
	cd $INSTALL_PATH
	tar -zxf $DOWNLOAD_PATH/apache-hive-2.3.5-bin.tar.gz -C $INSTALL_PATH/
	mv apache-hive-2.3.5-bin hive
	cd $INSTALL_PATH/hive/conf
	cp hive-default.xml.template hive-default.xml
	\cp -f $WORKSPACE/streamsets/scripts/conf/hive-site.xml $INSTALL_PATH/hive/conf/
	cp $INSTALL_PATH/hive/conf/hive-site.xml $INSTALL_PATH/hadoop/etc/hadoop/
	echo 'export HIVE_HOME=/usr/local/hive' >> /etc/profile
	echo 'export PATH=$PATH:$HIVE_HOME/bin' >> /etc/profile
	env_enable
}

install_mysql_connector_java()
{
	cd $INSTALL_PATH
	tar -zxf $DOWNLOAD_PATH/mysql-connector-java-5.1.40.tar.gz -C $INSTALL_PATH/
	cd $INSTALL_PATH/mysql-connector-java-5.1.40
	cp mysql-connector-java-5.1.40-bin.jar $INSTALL_PATH//hive/lib
}

start_hive()
{
	cd $INSTALL_PATH/hive/bin
	./schematool -dbType mysql -initSchema
	hive -e "create database test;exit;"
	nohup hiveserver2 &
}

install_streamsets()
{
	rpmdev-setuptree
	cp $WORKSPACE/streamsets/spec/streamsets.spec ~/rpmbuild/SPECS/
	cd ~/rpmbuild/SPECS
	spectool -g -R streamsets.spec
	rpmbuild -bb --quiet streamsets.spec
	
	if [[ $STRUCTURE =~ "x86_64" ]];then
		cd ~/rpmbuild/RPMS/x86_64
		yum -y install streamsets-3.3.0-1.x86_64.rpm
	elif [[ $STRUCTURE =~ "aarch64" ]];then
		cd ~/rpmbuild/RPMS/aarch64
		yum -y install streamsets-3.3.0-1.aarch64.rpm		
	else
		echo "System architecture not found, please try to install manually!!!"
	fi	
	
	cd $INSTALL_PATH/streamsets/
	mkdir sdc-extras
	\cp -f $WORKSPACE/streamsets/scripts/conf/sdc-security.policy $INSTALL_PATH/streamsets/etc/
	\cp -f $WORKSPACE/streamsets/scripts/conf/config /etc/selinux/
	echo -e 'sdc soft nofile 3276' >> /etc/security/limits.conf
	echo -e 'sdc hard nofile 32768' >> /etc/security/limits.conf
	systemctl stop firewalld.service
	echo 'ulimit -HSn 32768' >> /etc/profile
	echo 'ulimit -n 65535' >> /etc/profile
	echo 'export STREAMSETS_LIBRARIES_EXTRA_DIR=/usr/local/streamsets/sdc-extras/' >> /etc/profile
	env_enable
}

start_streamsets()
{
	cd $INSTALL_PATH/streamsets/bin
	./streamsets dc &
}

main()
{
    install_depends
	download_pkg
	install_jdk
	install_hadoop
	start_hadoop
	install_mysql
	start_mysql
	install_zookeeper
	start_zookeeper
	install_kafka
	start_kafka
	install_hive
	install_mysql_connector_java
	start_hive
	install_streamsets
	start_streamsets
}

main